# 爬取微博数据

import requests
import json
import time
from urllib import parse
import re
from kafka import KafkaProducer

# 创建kafka生产者
producer = KafkaProducer(bootstrap_servers="master:9092,node1:9092,node2:9092")

print("创建kafka连接成功")

word = "英雄联盟"

baseUrl = "https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D60%26q%3D$0%26t%3D10&isnewpage=1&extparam=cate%3D0%26pos%3D19%26realpos%3D19%26flag%3D0%26filter_type%3Drealtimehot%26c_type%3D31%26display_time%3D1600307254&luicode=10000011&lfid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&page_type=searchall&page=$1"

baseCommentUrl = "https://m.weibo.cn/comments/hotflow?id=$0&mid=$1&max_id_type=0"

for i in range(100):

    try:

        time.sleep(3)

        url = baseUrl.replace("$1", str(i)).replace("$0", parse.quote(word))

        result = requests.get(url)

        jsonObj = json.loads(result.text)

        for card in jsonObj["data"]["cards"]:
            mblog = card["mblog"]

            # 评论人数
            comments_count = mblog["comments_count"]

            # 发表时间
            created_at = mblog["created_at"]

            # 微博编号
            id = mblog["id"]

            # 微博内容
            text = mblog["text"]

            re_h = re.compile('</?\w+[^>]*>')  # 去掉HTML标签
            text = re_h.sub("", text)

            # 话题来演
            source = mblog["source"]

            # 转发人数
            reposts_count = mblog["reposts_count"]

            # 点赞人数
            attitudes_count = mblog["attitudes_count"]

            user = mblog["user"]

            user_id = user["id"]

            weibo = {}

            weibo["id"] = id
            weibo["comments_count"] = comments_count
            weibo["created_at"] = created_at
            weibo["source"] = source
            weibo["reposts_count"] = reposts_count
            weibo["attitudes_count"] = attitudes_count
            weibo["text"] = text
            weibo["user_id"] = user_id

            weibo_json = json.dumps(weibo, ensure_ascii=False)

            # 将微博数据发送到kafka
            producer.send("ods_kafka_weibo", value=weibo_json.encode("utf-8"))

            gender = user["gender"]

            screen_name = user["screen_name"]

            followers_count = user["followers_count"]

            follow_count = user["follow_count"]

            user_map = {}
            user_map["id"] = user_id
            user_map["gender"] = gender
            user_map["screen_name"] = screen_name
            user_map["followers_count"] = followers_count
            user_map["follow_count"] = follow_count

            user_json = json.dumps(user_map, ensure_ascii=False)

            # 将用户数据发送到kafka
            producer.send("ods_kafka_user", value=user_json.encode("utf-8"))

            mid = mblog["mid"]

            # 构建评价的url
            commentUrl = baseCommentUrl.replace("$0", id).replace("$1", mid)

            commentResult = requests.get(commentUrl)

            conmmentJson = json.loads(commentResult.text)

            for comment in conmmentJson["data"]["data"]:
                comment_text = comment["text"]

                # 取出html标签
                comment_text = re_h.sub("", comment_text)

                comment_time = comment["created_at"]

                comment_id = comment["id"]

                like_count = comment["like_count"]

                comment_user = comment["user"]

                # 取出评论用户
                user_id = comment_user["id"]

                comment_map = {}
                comment_map["comment_id"] = comment_id
                comment_map["comment_time"] = comment_time
                comment_map["like_count"] = like_count
                comment_map["comment_text"] = comment_text
                comment_map["user_id"] = user_id
                comment_map["weibo_id"] = id

                comment_json = json.dumps(comment_map, ensure_ascii=False)

                # 将评论的数据发送到afka
                producer.send("ods_kafka_comment", value=comment_json.encode("utf-8"))

                gender = comment_user["gender"]

                screen_name = comment_user["screen_name"]

                followers_count = comment_user["followers_count"]

                follow_count = comment_user["follow_count"]

                user_map = {}
                user_map["id"] = user_id
                user_map["gender"] = gender
                user_map["screen_name"] = screen_name
                user_map["followers_count"] = followers_count
                user_map["follow_count"] = follow_count

                user_json = json.dumps(user_map, ensure_ascii=False)

                # 将用户数据发送到kafka
                producer.send("ods_kafka_user", value=user_json.encode("utf-8"))

    except:
        print("连接超时")

    print(r"爬取第%d成功" % i)
